// ©2013-2016 Cameron Desrochers.
// Distributed under the simplified BSD license (see the license file that
// should have come with this header).
// Uses Jeff Preshing's semaphore implementation (under the terms of its
// separate zlib license, embedded below).

#pragma once

// Provides portable (VC++2010+, Intel ICC 13, GCC 4.7+, and anything C++11 compliant) implementation
// of low-level memory barriers, plus a few semi-portable utility macros (for inlining and alignment).
// Also has a basic atomic type (limited to hardware-supported atomics with no memory ordering guarantees).
// Uses the AE_* prefix for macros (historical reasons), and the "moodycamel" namespace for symbols.

#include <cassert>
#include <type_traits>
#include <cerrno>
#include <cstdint>
#include <ctime>

// Platform detection
#if defined(__INTEL_COMPILER)
#define AE_ICC
#elif defined(_MSC_VER)
#define AE_VCPP
#elif defined(__GNUC__)
#define AE_GCC
#endif

#if defined(_M_IA64) || defined(__ia64__)
#define AE_ARCH_IA64
#elif defined(_WIN64) || defined(__amd64__) || defined(_M_X64) || defined(__x86_64__)
#define AE_ARCH_X64
#elif defined(_M_IX86) || defined(__i386__)
#define AE_ARCH_X86
#elif defined(_M_PPC) || defined(__powerpc__)
#define AE_ARCH_PPC
#else
#define AE_ARCH_UNKNOWN
#endif

// AE_UNUSED
#define AE_UNUSED(x) ((void)x)

// AE_FORCEINLINE
#if defined(AE_VCPP) || defined(AE_ICC)
#define AE_FORCEINLINE __forceinline
#elif defined(AE_GCC)
//#define AE_FORCEINLINE __attribute__((always_inline))
#define AE_FORCEINLINE inline
#else
#define AE_FORCEINLINE inline
#endif

// AE_ALIGN
#if defined(AE_VCPP) || defined(AE_ICC)
#define AE_ALIGN(x) __declspec(align(x))
#elif defined(AE_GCC)
#define AE_ALIGN(x) __attribute__((aligned(x)))
#else
// Assume GCC compliant syntax...
#define AE_ALIGN(x) __attribute__((aligned(x)))
#endif

// Portable atomic fences implemented below:

namespace moodycamel {

enum memory_order {
	memory_order_relaxed,
	memory_order_acquire,
	memory_order_release,
	memory_order_acq_rel,
	memory_order_seq_cst,

	// memory_order_sync: Forces a full sync:
	// #LoadLoad, #LoadStore, #StoreStore, and most significantly, #StoreLoad
	memory_order_sync = memory_order_seq_cst
};

} // end namespace moodycamel

#if (defined(AE_VCPP) && (_MSC_VER < 1700 || defined(__cplusplus_cli))) || defined(AE_ICC)
// VS2010 and ICC13 don't support std::atomic_*_fence, implement our own fences

#include <intrin.h>

#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86)
#define AeFullSync _mm_mfence
#define AeLiteSync _mm_mfence
#elif defined(AE_ARCH_IA64)
#define AeFullSync __mf
#define AeLiteSync __mf
#elif defined(AE_ARCH_PPC)
#include <ppcintrinsics.h>
#define AeFullSync __sync
#define AeLiteSync __lwsync
#endif

#ifdef AE_VCPP
#pragma warning(push)
#pragma warning(disable: 4365)		// Disable erroneous 'conversion from long to unsigned int, signed/unsigned mismatch' error when using `assert`
#ifdef __cplusplus_cli
#pragma managed(push, off)
#endif
#endif

namespace moodycamel {

	AE_FORCEINLINE void compiler_fence(memory_order order)
	{
		switch (order) {
			case memory_order_relaxed: break;
			case memory_order_acquire: _ReadBarrier(); break;
			case memory_order_release: _WriteBarrier(); break;
			case memory_order_acq_rel: _ReadWriteBarrier(); break;
			case memory_order_seq_cst: _ReadWriteBarrier(); break;
			default: assert(false);
		}
	}

// x86/x64 have a strong memory model -- all loads and stores have
// acquire and release semantics automatically (so only need compiler
// barriers for those).
#if defined(AE_ARCH_X86) || defined(AE_ARCH_X64)
	AE_FORCEINLINE void fence(memory_order order)
	{
		switch (order) {
			case memory_order_relaxed: break;
			case memory_order_acquire: _ReadBarrier(); break;
			case memory_order_release: _WriteBarrier(); break;
			case memory_order_acq_rel: _ReadWriteBarrier(); break;
			case memory_order_seq_cst:
			_ReadWriteBarrier();
			AeFullSync();
			_ReadWriteBarrier();
			break;
			default: assert(false);
		}
	}
#else
	AE_FORCEINLINE void fence(memory_order order)
	{
		// Non-specialized arch, use heavier memory barriers everywhere just in case :-(
		switch (order) {
			case memory_order_relaxed:
			break;
			case memory_order_acquire:
			_ReadBarrier();
			AeLiteSync();
			_ReadBarrier();
			break;
			case memory_order_release:
			_WriteBarrier();
			AeLiteSync();
			_WriteBarrier();
			break;
			case memory_order_acq_rel:
			_ReadWriteBarrier();
			AeLiteSync();
			_ReadWriteBarrier();
			break;
			case memory_order_seq_cst:
			_ReadWriteBarrier();
			AeFullSync();
			_ReadWriteBarrier();
			break;
			default: assert(false);
		}
	}
#endif
} // end namespace moodycamel
#else
// Use standard library of atomics
#include <atomic>

namespace moodycamel {

AE_FORCEINLINE void compiler_fence(memory_order order) {
	switch (order) {
	case memory_order_relaxed:
		break;
	case memory_order_acquire:
		std::atomic_signal_fence(std::memory_order_acquire);
		break;
	case memory_order_release:
		std::atomic_signal_fence(std::memory_order_release);
		break;
	case memory_order_acq_rel:
		std::atomic_signal_fence(std::memory_order_acq_rel);
		break;
	case memory_order_seq_cst:
		std::atomic_signal_fence(std::memory_order_seq_cst);
		break;
	default:
		assert(false);
	}
}

AE_FORCEINLINE void fence(memory_order order) {
	switch (order) {
	case memory_order_relaxed:
		break;
	case memory_order_acquire:
		std::atomic_thread_fence(std::memory_order_acquire);
		break;
	case memory_order_release:
		std::atomic_thread_fence(std::memory_order_release);
		break;
	case memory_order_acq_rel:
		std::atomic_thread_fence(std::memory_order_acq_rel);
		break;
	case memory_order_seq_cst:
		std::atomic_thread_fence(std::memory_order_seq_cst);
		break;
	default:
		assert(false);
	}
}

} // end namespace moodycamel

#endif

#if !defined(AE_VCPP) || (_MSC_VER >= 1700 && !defined(__cplusplus_cli))
#define AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC
#endif

#ifdef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC
#include <atomic>
#endif
#include <utility>

// WARNING: *NOT* A REPLACEMENT FOR std::atomic. READ CAREFULLY:
// Provides basic support for atomic variables -- no memory ordering guarantees are provided.
// The guarantee of atomicity is only made for types that already have atomic load and store guarantees
// at the hardware level -- on most platforms this generally means aligned pointers and integers (only).
namespace moodycamel {
template<typename T>
class weak_atomic {
public:
	weak_atomic() {
	}
#ifdef AE_VCPP
#pragma warning(disable: 4100)		// Get rid of (erroneous) 'unreferenced formal parameter' warning
#endif
	template<typename U> weak_atomic(U&& x) : value(std::forward<U>(x)) {}
#ifdef __cplusplus_cli
			// Work around bug with universal reference/nullptr combination that only appears when /clr is on
			weak_atomic(nullptr_t) : value(nullptr) {}
#endif
			weak_atomic(weak_atomic const& other) : value(other.value) {}
			weak_atomic(weak_atomic&& other) : value(std::move(other.value)) {}
#ifdef AE_VCPP
#pragma warning(default: 4100)
#endif

			AE_FORCEINLINE operator T() const {return load();}

#ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC
			template<typename U> AE_FORCEINLINE weak_atomic const& operator=(U&& x) {value = std::forward<U>(x); return *this;}
			AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other) {value = other.value; return *this;}

			AE_FORCEINLINE T load() const {return value;}

			AE_FORCEINLINE T fetch_add_acquire(T increment)
			{
#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86)
			if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment);
#if defined(_M_AMD64)
			else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment);
#endif
#else
#error Unsupported platform
#endif
			assert(false && "T must be either a 32 or 64 bit type");
			return value;
		}

		AE_FORCEINLINE T fetch_add_release(T increment)
		{
#if defined(AE_ARCH_X64) || defined(AE_ARCH_X86)
			if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment);
#if defined(_M_AMD64)
			else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment);
#endif
#else
#error Unsupported platform
#endif
			assert(false && "T must be either a 32 or 64 bit type");
			return value;
		}
#else
			template<typename U>
			AE_FORCEINLINE weak_atomic const& operator=(U&& x)
			{
				value.store(std::forward<U>(x), std::memory_order_relaxed);
				return *this;
			}

			AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other)
			{
				value.store(other.value.load(std::memory_order_relaxed), std::memory_order_relaxed);
				return *this;
			}

			AE_FORCEINLINE T load() const {return value.load(std::memory_order_relaxed);}

			AE_FORCEINLINE T fetch_add_acquire(T increment)
			{
				return value.fetch_add(increment, std::memory_order_acquire);
			}

			AE_FORCEINLINE T fetch_add_release(T increment)
			{
				return value.fetch_add(increment, std::memory_order_release);
			}
#endif

		private:
#ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC
			// No std::atomic support, but still need to circumvent compiler optimizations.
			// `volatile` will make memory access slow, but is guaranteed to be reliable.
			volatile T value;
#else
			std::atomic<T> value;
#endif
		};

		} // end namespace moodycamel

// Portable single-producer, single-consumer semaphore below:

#if defined(_WIN32)
// Avoid including windows.h in a header; we only need a handful of
// items, so we'll redeclare them here (this is relatively safe since
// the API generally has to remain stable between Windows versions).
// I know this is an ugly hack but it still beats polluting the global
// namespace with thousands of generic names or adding a .cpp for nothing.
			extern "C" {
				struct _SECURITY_ATTRIBUTES;
				__declspec(dllimport) void* __stdcall CreateSemaphoreW(_SECURITY_ATTRIBUTES* lpSemaphoreAttributes, long lInitialCount, long lMaximumCount, const wchar_t* lpName);
				__declspec(dllimport) int __stdcall CloseHandle(void* hObject);
				__declspec(dllimport) unsigned long __stdcall WaitForSingleObject(void* hHandle, unsigned long dwMilliseconds);
				__declspec(dllimport) int __stdcall ReleaseSemaphore(void* hSemaphore, long lReleaseCount, long* lpPreviousCount);
			}
#elif defined(__MACH__)
#include <mach/mach.h>
#elif defined(__unix__)
#include <semaphore.h>
#endif

			namespace moodycamel {
			// Code in the spsc_sema namespace below is an adaptation of Jeff Preshing's
			// portable + lightweight semaphore implementations, originally from
			// https://github.com/preshing/cpp11-on-multicore/blob/master/common/sema.h
			// LICENSE:
			// Copyright (c) 2015 Jeff Preshing
			//
			// This software is provided 'as-is', without any express or implied
			// warranty. In no event will the authors be held liable for any damages
			// arising from the use of this software.
			//
			// Permission is granted to anyone to use this software for any purpose,
			// including commercial applications, and to alter it and redistribute it
			// freely, subject to the following restrictions:
			//
			// 1. The origin of this software must not be misrepresented; you must not
			//    claim that you wrote the original software. If you use this software
			//    in a product, an acknowledgement in the product documentation would be
			//    appreciated but is not required.
			// 2. Altered source versions must be plainly marked as such, and must not be
			//    misrepresented as being the original software.
			// 3. This notice may not be removed or altered from any source distribution.
			namespace spsc_sema {
#if defined(_WIN32)
			class Semaphore
			{
			private:
				void* m_hSema;

				Semaphore(const Semaphore& other);
				Semaphore& operator=(const Semaphore& other);

			public:
				Semaphore(int initialCount = 0)
				{
					assert(initialCount >= 0);
					const long maxLong = 0x7fffffff;
					m_hSema = CreateSemaphoreW(nullptr, initialCount, maxLong, nullptr);
				}

				~Semaphore()
				{
					CloseHandle(m_hSema);
				}

				void wait()
				{
					const unsigned long infinite = 0xffffffff;
					WaitForSingleObject(m_hSema, infinite);
				}

				bool try_wait()
				{
					const unsigned long RC_WAIT_TIMEOUT = 0x00000102;
					return WaitForSingleObject(m_hSema, 0) != RC_WAIT_TIMEOUT;
				}

				bool timed_wait(std::uint64_t usecs)
				{
					const unsigned long RC_WAIT_TIMEOUT = 0x00000102;
					return WaitForSingleObject(m_hSema, (unsigned long)(usecs / 1000)) != RC_WAIT_TIMEOUT;
				}

				void signal(int count = 1)
				{
					ReleaseSemaphore(m_hSema, count, nullptr);
				}
			};
#elif defined(__MACH__)
			//---------------------------------------------------------
			// Semaphore (Apple iOS and OSX)
			// Can't use POSIX semaphores due to http://lists.apple.com/archives/darwin-kernel/2009/Apr/msg00010.html
			//---------------------------------------------------------
			class Semaphore
			{
			private:
				semaphore_t m_sema;

				Semaphore(const Semaphore& other);
				Semaphore& operator=(const Semaphore& other);

			public:
				Semaphore(int initialCount = 0)
				{
					assert(initialCount >= 0);
					semaphore_create(mach_task_self(), &m_sema, SYNC_POLICY_FIFO, initialCount);
				}

				~Semaphore()
				{
					semaphore_destroy(mach_task_self(), m_sema);
				}

				void wait()
				{
					semaphore_wait(m_sema);
				}

				bool try_wait()
				{
					return timed_wait(0);
				}

				bool timed_wait(std::int64_t timeout_usecs)
				{
					mach_timespec_t ts;
					ts.tv_sec = timeout_usecs / 1000000;
					ts.tv_nsec = (timeout_usecs % 1000000) * 1000;

					// added in OSX 10.10: https://developer.apple.com/library/prerelease/mac/documentation/General/Reference/APIDiffsMacOSX10_10SeedDiff/modules/Darwin.html
					kern_return_t rc = semaphore_timedwait(m_sema, ts);

					return rc != KERN_OPERATION_TIMED_OUT;
				}

				void signal()
				{
					semaphore_signal(m_sema);
				}

				void signal(int count)
				{
					while (count-- > 0)
					{
						semaphore_signal(m_sema);
					}
				}
			};
#elif defined(__unix__)
			//---------------------------------------------------------
			// Semaphore (POSIX, Linux)
			//---------------------------------------------------------
			class Semaphore {
			private:
				sem_t m_sema;

				Semaphore(const Semaphore& other);
				Semaphore& operator=(const Semaphore& other);

			public:
				Semaphore(int initialCount = 0) {
					assert(initialCount >= 0);
					sem_init(&m_sema, 0, initialCount);
				}

				~Semaphore() {
					sem_destroy(&m_sema);
				}

				void wait() {
					// http://stackoverflow.com/questions/2013181/gdb-causes-sem-wait-to-fail-with-eintr-error
					int rc;
					do {
						rc = sem_wait(&m_sema);
					} while (rc == -1 && errno == EINTR);
				}

				bool try_wait() {
					int rc;
					do {
						rc = sem_trywait(&m_sema);
					} while (rc == -1 && errno == EINTR);
					return !(rc == -1 && errno == EAGAIN);
				}

				bool timed_wait(std::uint64_t usecs) {
					struct timespec ts;
					const int usecs_in_1_sec = 1000000;
					const int nsecs_in_1_sec = 1000000000;
					clock_gettime(CLOCK_REALTIME, &ts);
					ts.tv_sec += usecs / usecs_in_1_sec;
					ts.tv_nsec += (usecs % usecs_in_1_sec) * 1000;
					// sem_timedwait bombs if you have more than 1e9 in tv_nsec
					// so we have to clean things up before passing it in
					if (ts.tv_nsec > nsecs_in_1_sec) {
						ts.tv_nsec -= nsecs_in_1_sec;
						++ts.tv_sec;
					}

					int rc;
					do {
						rc = sem_timedwait(&m_sema, &ts);
					} while (rc == -1 && errno == EINTR);
					return !(rc == -1 && errno == ETIMEDOUT);
				}

				void signal() {
					sem_post(&m_sema);
				}

				void signal(int count) {
					while (count-- > 0) {
						sem_post(&m_sema);
					}
				}
			};
#else
#error Unsupported platform! (No semaphore wrapper available)
#endif

			//---------------------------------------------------------
			// LightweightSemaphore
			//---------------------------------------------------------
			class LightweightSemaphore {
			public:
				typedef std::make_signed<std::size_t>::type ssize_t;

			private:
				weak_atomic<ssize_t> m_count;
				Semaphore m_sema;

				bool waitWithPartialSpinning(std::int64_t timeout_usecs = -1) {
					ssize_t oldCount;
					// Is there a better way to set the initial spin count?
					// If we lower it to 1000, testBenaphore becomes 15x slower on my Core i7-5930K Windows PC,
					// as threads start hitting the kernel semaphore.
					int spin = 10000;
					while (--spin >= 0) {
						if (m_count.load() > 0) {
							m_count.fetch_add_acquire(-1);
							return true;
						}
						compiler_fence(memory_order_acquire); // Prevent the compiler from collapsing the loop.
					}
					oldCount = m_count.fetch_add_acquire(-1);
					if (oldCount > 0)
						return true;
					if (timeout_usecs < 0) {
						m_sema.wait();
						return true;
					}
					if (m_sema.timed_wait(timeout_usecs))
						return true;
					// At this point, we've timed out waiting for the semaphore, but the
					// count is still decremented indicating we may still be waiting on
					// it. So we have to re-adjust the count, but only if the semaphore
					// wasn't signaled enough times for us too since then. If it was, we
					// need to release the semaphore too.
					while (true) {
						oldCount = m_count.fetch_add_release(1);
						if (oldCount < 0)
							return false; // successfully restored things to the way they were
						// Oh, the producer thread just signaled the semaphore after all. Try again:
						oldCount = m_count.fetch_add_acquire(-1);
						if (oldCount > 0 && m_sema.try_wait())
							return true;
					}
				}

			public:
				LightweightSemaphore(ssize_t initialCount = 0) :
						m_count(initialCount) {
					assert(initialCount >= 0);
				}

				bool tryWait() {
					if (m_count.load() > 0) {
						m_count.fetch_add_acquire(-1);
						return true;
					}
					return false;
				}

				void wait() {
					if (!tryWait())
						waitWithPartialSpinning();
				}

				bool wait(std::int64_t timeout_usecs) {
					return tryWait() || waitWithPartialSpinning(timeout_usecs);
				}

				void signal(ssize_t count = 1) {
					assert(count >= 0);
					ssize_t oldCount = m_count.fetch_add_release(count);
					assert(oldCount >= -1);
					if (oldCount < 0) {
						m_sema.signal(1);
					}
				}

				ssize_t availableApprox() const {
					ssize_t count = m_count.load();
					return count > 0 ? count : 0;
				}
			};
			} // end namespace spsc_sema
			} // end namespace moodycamel

#if defined(AE_VCPP) && (_MSC_VER < 1700 || defined(__cplusplus_cli))
#pragma warning(pop)
#ifdef __cplusplus_cli
#pragma managed(pop)
#endif
#endif
